import re
import pdftotext

def text_to_doc(filename = None):
    if not filename:
        filename = "/Users/avinash.v/Downloads/sample.txt"
    new_lines = []
    last_new_line = -2
    line_count = 0
    with open(filename,"r") as f:
        line = f.readline()
        while line:
            try:
                line_count += 1
                line = line.strip()
                all_upper = line.upper() == line
                if len(line) > 0 :

                    # remove non ascii chars
                    if not all_upper:
                        line = re.sub(r'[^\x00-\x7F]', ' ', line).strip()

                        # a heuristic to look for paragraph boundary
                        # if the pervios line was a new line
                        if last_new_line == line_count -1:
                            # if len(line) >= 10:
                            new_lines.append("::" + line)
                            print ("para start ---> " + line)

                        else:
                            new_lines.append(line)
                        # continue
                        #     next_line = f.readline()
                        #     l = len(next_line)
                        #     if len(next_line.strip()) == 0 or next_line.strip().endswith(":") or len:
                        #         # beginning of new para
                        #         new_lines.append("::" + line)
                        #         print ("---> " + line)
                        #     # we will have to set the file point back by l
                        #     f.seek(-l,1)
                        # else:
                        #     new_lines.append(line)
                else:
                    last_new_line = line_count
                line = f.readline()
            except StandardError as e:
                print("Error processing file " + filename + "\nline: " + line)

    # new_lines = re.split('\.\s+',' '.join(new_lines))
    paragraphs = re.split('::','\n'.join(new_lines))
    # new_lines = re.split('[a-z0-9]+\s+',' '.join(new_lines))
    print("-----")
    for p in paragraphs:
        print(p)

    last_paragraph = -1
    final_list_of_paragraphs = []
    tmp_para_list = []
    for i in range(len(paragraphs)):
        p = paragraphs[i]
        if not p.strip().endswith("."):
            # its not a valid paragraph
            tmp_para_list.append(p)
        else:
            # pop paras from tmp_list_para and join them and add it to p
            # add this to final_list_of_paragraphs
            tmp_ps = " ".join(tmp_para_list) + " " + p
            tmp_para_list = []
            final_list_of_paragraphs.append(tmp_ps)

    print("-----")
    print("FINAL")
    for i in range(len(final_list_of_paragraphs)):
        final_list_of_paragraphs[i] = "\n".join(re.split('\n+',final_list_of_paragraphs[i]))
        print(final_list_of_paragraphs[i])
        print("****")

    output_filename = filename.split(".txt")[0] + '_docs.txt'
    print(output_filename)


    with open(output_filename, "w") as f:
        for p in final_list_of_paragraphs:
            f.write(p+"\n")

if __name__ == '__main__':
    filename = "/Users/avinash.v/Downloads/sample2.txt"
    filename = "/Users/avinash.v/Downloads/sample4.txt"
    text_to_doc(filename)